import time
import requests
import json
import pymongo
from concurrent.futures import ThreadPoolExecutor


def fetch_and_insert(page, session, collection):
    url = f"https://www.governbond.org.cn:4443/api/loadBondData.action?timeStamp=1744278770234&dataType=ZQFXLISTBYAD&adList=&adCode=87&zqlx=&year=&fxfs=&qxr=&fxqx=&zqCode=&zqName=&page={page}&pageSize=10"
    headers = {
        "user-agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Mobile Safari/537.36"
    }
    try:
        response = session.get(url, headers=headers)
        response.raise_for_status()
        html = response.json()
        data_to_insert = []
        for zq in html['data']:
            data_to_insert.append({
                "债券名称": zq['ZQ_NAME'],
                "债券编码": zq['ZQ_CODE'],
                "发行地区": zq['AD_NAME'],
                "发行时间": zq['ZQ_FXTIME'],
                "发行期限": zq['ZQQX_NAME'],
                "发行规模": zq['FX_AMT'],
                "利率": zq['LL'],
                "付息方式": zq['FXFS'],
                "起息日": zq['QX_DATE']
            })
            print(f"正在爬取第{page}页债券{zq['ZQ_CODE']}......")
        if data_to_insert:
            collection.insert_many(data_to_insert)
    except requests.RequestException as e:
        print(f"请求第 {page} 页失败: {e}")
    except json.JSONDecodeError as e:
        print(f"解析第 {page} 页的JSON数据失败: {e}")
    except pymongo.errors.PyMongoError as e:
        print(f"插入第 {page} 页数据到数据库失败: {e}")
    # 添加请求间隔
    time.sleep(0.1)


client = pymongo.MongoClient('mongodb://localhost:27017/')
db_name = '中国地方政府债券线程版'
client.drop_database(db_name)
db = client[db_name]
collection = db['data']
session = requests.Session()

with ThreadPoolExecutor(max_workers=50) as executor:
    pages = range(1, 1642)
    futures = [executor.submit(fetch_and_insert, page, session, collection) for page in pages]
    for future in futures:
        future.result()


session.close()
client.close()